<?php
set_time_limit(0);
include 'db.php';
$fp = fopen('tag-utf8.txt', 'r');
$ofp = fopen('tags.txt', 'wr+');

$line = '';
$index = 1;
$stm = time();
$mstm = microtime();
$insertWords = array();
while ($line = fgets($fp, 1024)){
	$index++;
	$line = clearLine($line);
	$words = explode(',', $line);
	if ($words){
		foreach ($words as $word){
			if (!$word) continue;
			$insertWords[] = $word;
			fwrite($ofp, $word . PHP_EOL);
		}
	}
	 
	 if (!($index % 1000)){
	 	$insert = array();
	 	foreach ($insertWords as $word){
	 		$insert[] = "(NULL, '{$word}')";
	 	}
	 	
	 	$sql = "insert into `words` (`id`,`word`) values" . implode(',', $insert);
	 	mysql_query($sql, $link);
	 	$insertWords = array();
	 	echo $index, ' line completed,time wasted:', time()-$stm, PHP_EOL;
	 }
}

function clearLine($line){
	trim($line, '\t\n\r,');
	$line = preg_replace("/[\/\|]+/", ',', $line);
	return $line;
}